﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using LingDong.ConfigProvider;
using LingDong.HtmlParser;
using NLog;

namespace LingDong.DataProvider
{
    /// <summary>
    /// Read data from SogouT
    /// SogouT is an internet corpus provided by sogou lab
    /// http://www.sogou.com/labs/dl/t.html
    /// Format: 
    /// <doc>
    /// <docno>Doc ID</docno>
    /// <url>Doc URL</url>
    /// Doc original content, including HTML tags
    /// </doc>
    /// </summary>
    public class SogouTDataReader : IDataReader
    {
        #region Contructors

        public SogouTDataReader()
        {
            logger = LogManager.GetCurrentClassLogger();
            // init encoding
            InitEncodingList();

            // init data file
            InitDataFileList();

            ReSetStreamReader();
        }

        #endregion

        
        #region Public Members

        /// <summary>
        /// Get next html data in sogou data file
        /// </summary>
        /// <returns></returns>
        public DataEntity GetNextData()
        {
            do
            {
                if (IsAllDataRead())
                    return null;

                if (reader.EndOfStream)
                    return ChangeToNextEncoding();

                DataEntity data = new DataEntity();
                try
                {
                    string line;
                    // skip blank lines
                    do
                    {
                        line = reader.ReadLine();
                    } while (String.IsNullOrEmpty(line) && !reader.EndOfStream);

                    if (reader.EndOfStream)
                        return ChangeToNextEncoding();

                    // doc id and url
                    string docid = reader.ReadLine();
                    string url = reader.ReadLine();
                    data.DocID = docid.Substring(7, docid.Length - 15);
                    data.Url = url.Substring(5, url.Length - 11);

                    // html content
                    StringBuilder content = new StringBuilder();
                    while (!reader.EndOfStream)
                    {
                        line = reader.ReadLine();
                        if (line == "</DOC>")
                            break;
                        content.AppendLine(line);
                    }
                    data.Content = content.ToString().Trim();
                    
                    Encoding declaredEncode = HtmlContent.GetDeclaredEncoding(data.Content);
                    // convert document encoding to index default encoding
                    //if (declaredEncode != defaultEncoding)
                    //    data.Content = ConvertEncoding(data.Content, declaredEncode);

                    // return only when this document is in current encoding
                    if (declaredEncode == EncodingList[currentEncodingIndex])
                    {
                        return data;
                    }
                }
                catch (Exception ex)
                {
                    logger.Error("Unexpeted format in sogouT data file: {0}", ex.Message);
                    throw;
                }
            } while (true);
        }

        /// <summary>
        /// Return if all data in sogou data file has been read
        /// </summary>
        /// <returns></returns>
        public bool IsAllDataRead()
        {
            return currentEncodingIndex >= EncodingList.Count && 
                currentDataFileIndex >= dataFileList.Count &&
                reader.EndOfStream;
        }

        /// <summary>
        /// Return true if the data only contains text
        /// false if the data contains HTML tags
        /// </summary>
        /// <returns></returns>
        public bool IsDataContainsOnlyText()
        {
            return false;
        }

        #endregion


        #region Private Methods

        private void InitEncodingList()
        {
            currentEncodingIndex = 0;
            EncodingList = new List<Encoding>();
            foreach (string encode in ConfigDataProvider.Default.SogouTDataEncode)
            {
                EncodingList.Add(Encoding.GetEncoding(encode));
            }

            defaultEncoding = Encoding.GetEncoding(ConfigIndex.Default.IndexDocEncode);
        }

        private void InitDataFileList()
        {
            currentDataFileIndex = 0;
            string path = ConfigDataProvider.Default.SogouTDataPath;
            // given path is a single file
            if (File.Exists(path))
            {
                dataFileList = new List<string>();
                dataFileList.Add(path);
            }
            // given path is a directory, add all files in it
            else if (Directory.Exists(path))
            {
                dataFileList = Directory.GetFiles(path, "*", SearchOption.AllDirectories).ToList();
            }
            else
            {
                throw new ArgumentException("path");
            }
        }

        private DataEntity ChangeToNextEncoding()
        {
            if (++currentEncodingIndex >= EncodingList.Count)
                return ChangeToNextFile();

            ReSetStreamReader();
            return GetNextData();
        }
        
        /// <summary>
        /// After all encodings have been tried, read next file
        /// </summary>
        /// <returns></returns>
        private DataEntity ChangeToNextFile()
        {
            if (++currentDataFileIndex >= dataFileList.Count)
                return null;

            logger.Debug("SogouT data reader, reading file: {0}", dataFileList[currentDataFileIndex]);
            currentEncodingIndex = 0;
            ReSetStreamReader();
            return GetNextData();
        }

        private void ReSetStreamReader()
        {
            if (reader != null)
                reader.Dispose();

            try
            {
                reader = new StreamReader(dataFileList[currentDataFileIndex],
                                          EncodingList[currentEncodingIndex]);
            }
            catch (Exception ex)
            {
                logger.Error("Cannot open Sogoug data file: {0}", ex.Message);
                throw;
            }
        }

        private string ConvertEncoding(string content, Encoding encoding)
        {
            byte[] curBytes = encoding.GetBytes(content);
            byte[] defaultBytes = Encoding.Convert(encoding, defaultEncoding, curBytes);
            return defaultEncoding.GetString(defaultBytes);
        }

        #endregion


        #region Private Members

        private StreamReader reader;

        private int currentEncodingIndex;
        private List<Encoding> EncodingList;

        private int currentDataFileIndex;
        private List<string> dataFileList;

        private Encoding defaultEncoding;
        private Logger logger;

        #endregion
        

        #region Disposable related

        public void Dispose()
        {
            Dispose(true);
            GC.SuppressFinalize(this);
        }

        protected virtual void Dispose(bool disposing)
        {
            // protect from being called multiple times.
            if (disposed)
            {
                return;
            }

            if (disposing)
            {
                // clean up all managed resources.
                if (reader != null)
                {
                    reader.Dispose();
                }
            }

            disposed = true;
        }

        private bool disposed = false;

        #endregion
    }
}
